





<!DOCTYPE html>
<html class="writer-html5" lang="zh-CN" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Relay TensorRT Integration &mdash; tvm 0.8.dev1982 文档</title>
  

  
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/tlcpack_theme.css" type="text/css" />

  
  
    <link rel="shortcut icon" href="../../_static/tvm-logo-square.png"/>
  

  
  
  
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
        <script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
        <script src="../../_static/jquery.js"></script>
        <script src="../../_static/underscore.js"></script>
        <script src="../../_static/doctools.js"></script>
        <script src="../../_static/translations.js"></script>
    
    <script type="text/javascript" src="../../_static/js/theme.js"></script>

    
    <script type="text/javascript" src="../../_static/js/tlcpack_theme.js"></script>
    <link rel="index" title="索引" href="../../genindex.html" />
    <link rel="search" title="搜索" href="../../search.html" />
    <link rel="next" title="Vitis AI Integration" href="vitis_ai.html" />
    <link rel="prev" title="Relay Arm® Compute Library Integration" href="arm_compute_lib.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    
<header class="header">
    <div class="innercontainer">
      <div class="headerInner d-flex justify-content-between align-items-center">
          <div class="headerLogo">
               <a href="https://tvm.apache.org/"><img src=https://tvm.apache.org/assets/images/logo.svg alt="logo"></a>
          </div>

          <div id="headMenu" class="headerNav">
            <button type="button" id="closeHeadMenu" class="navCloseBtn"><img src="../../_static/img/close-icon.svg" alt="Close"></button>
             <ul class="nav">
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/community>Community</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/download>Download</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/vta>VTA</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/blog>Blog</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/docs>Docs</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmconf.org>Conference</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://github.com/apache/tvm/>Github</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmchinese.github.io/declaration_zh_CN.html>About-Translators</a>
                </li>
             </ul>
               <div class="responsivetlcdropdown">
                 <button type="button" class="btn-link">
                   ASF
                 </button>
                 <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                 </ul>
               </div>
          </div>
            <div class="responsiveMenuIcon">
              <button type="button" id="menuBtn" class="btn-menu"><img src="../../_static/img/menu-icon.svg" alt="Menu Icon"></button>
            </div>

            <div class="tlcDropdown">
              <div class="dropdown">
                <button type="button" class="btn-link dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                  ASF
                </button>
                <div class="dropdown-menu dropdown-menu-right">
                  <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                  </ul>
                </div>
              </div>
          </div>
       </div>
    </div>
 </header>
 
    <nav data-toggle="wy-nav-shift" class="wy-nav-side fixed">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../index.html">
          

          
            
            <img src="../../_static/tvm-logo-small.png" class="logo" alt="Logo"/>
          
          </a>

          
            
            
                <div class="version">
                  0.8.dev1982
                </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption" role="heading"><span class="caption-text">如何开始</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../install/index.html">安装 TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../contribute/index.html">贡献者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">用户引导</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../tutorial/index.html">User Tutorial</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">How To Guides</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../compile_models/index.html">编译深度学习模型</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html">TVM 部署模型和集成</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="index.html#build-the-tvm-runtime-library">构建 TVM 运行 runtime 库</a></li>
<li class="toctree-l3"><a class="reference internal" href="index.html#cross-compile-the-tvm-runtime-for-other-architectures">为其它架构交叉编译TVM runtime</a></li>
<li class="toctree-l3"><a class="reference internal" href="index.html#optimize-and-tune-models-for-target-devices">针对目标设备优化和调整模型</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="index.html#deploy-optimized-model-on-target-devices">在目标设备上部署优化的模型</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="cpp_deploy.html">Deploy TVM Module using C++ API</a></li>
<li class="toctree-l4"><a class="reference internal" href="android.html">部署到安卓</a></li>
<li class="toctree-l4"><a class="reference internal" href="integrate.html">Integrate TVM into Your Project</a></li>
<li class="toctree-l4"><a class="reference internal" href="hls.html">HLS Backend Example</a></li>
<li class="toctree-l4"><a class="reference internal" href="arm_compute_lib.html">Relay Arm<sup>®</sup> Compute Library Integration</a></li>
<li class="toctree-l4 current"><a class="current reference internal" href="#">Relay TensorRT Integration</a></li>
<li class="toctree-l4"><a class="reference internal" href="vitis_ai.html">Vitis AI Integration</a></li>
<li class="toctree-l4"><a class="reference internal" href="bnns.html">Relay BNNS Integration</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="index.html#additional-deployment-how-tos">其他部署方式</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../work_with_relay/index.html">Work With Relay</a></li>
<li class="toctree-l2"><a class="reference internal" href="../work_with_schedules/index.html">Work With Tensor Expression and Schedules</a></li>
<li class="toctree-l2"><a class="reference internal" href="../optimize_operators/index.html">优化张量算子</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tune_with_autotvm/index.html">Auto-Tune with Templates and AutoTVM</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tune_with_autoscheduler/index.html">Use AutoScheduler for Template-Free Scheduling</a></li>
<li class="toctree-l2"><a class="reference internal" href="../work_with_microtvm/index.html">Work With microTVM</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extend_tvm/index.html">Extend TVM</a></li>
<li class="toctree-l2"><a class="reference internal" href="../profile/index.html">Profile Models</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../errors.html">Handle TVM Errors</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../faq.html">常见提问</a></li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">开发者引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../dev/tutorial/index.html">Developer Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/how_to/how_to.html">开发者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">架构指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../arch/index.html">Design and Architecture</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">主题引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../topic/microtvm/index.html">microTVM：裸机使用TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../topic/vta/index.html">VTA: Versatile Tensor Accelerator</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">参考指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../reference/langref/index.html">语言参考</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../reference/api/python/index.html">Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../reference/api/links.html">Other APIs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../reference/publications.html">Publications</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../genindex.html">索引</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
      
      <nav class="wy-nav-top" aria-label="top navigation" data-toggle="wy-nav-top">
        
            <div class="togglemenu">

            </div>
            <div class="nav-content">
              <!-- tvm -->
              Table of content
            </div>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        

          




















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../index.html">Docs</a> <span class="br-arrow">></span></li>
        
          <li><a href="../index.html">How To Guides</a> <span class="br-arrow">></span></li>
        
          <li><a href="index.html">TVM 部署模型和集成</a> <span class="br-arrow">></span></li>
        
      <li>Relay TensorRT Integration</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../../_sources/how_to/deploy/tensorrt.rst.txt" rel="nofollow"> <img src="../../_static//img/source.svg" alt="viewsource"/></a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="relay-tensorrt-integration">
<h1>Relay TensorRT Integration<a class="headerlink" href="#relay-tensorrt-integration" title="永久链接至标题">¶</a></h1>
<p><strong>Author</strong>: <a class="reference external" href="https://github.com/trevor-m">Trevor Morris</a></p>
<div class="section" id="introduction">
<h2>介绍<a class="headerlink" href="#introduction" title="永久链接至标题">¶</a></h2>
<p>NVIDIA TensorRT is a library for optimized deep learning inference. This integration will offload as
many operators as possible from Relay to TensorRT, providing a performance boost on NVIDIA GPUs
without the need to tune schedules.</p>
<p>This guide will demonstrate how to install TensorRT and build TVM with TensorRT BYOC and runtime
enabled. It will also provide example code to compile and run a ResNet-18 model using TensorRT and
how to configure the compilation and runtime settings. Finally, we document the supported operators
and how to extend the integration to support other operators.</p>
</div>
<div class="section" id="installing-tensorrt">
<h2>Installing TensorRT<a class="headerlink" href="#installing-tensorrt" title="永久链接至标题">¶</a></h2>
<p>In order to download TensorRT, you will need to create an NVIDIA Developer program account. Please
see NVIDIA’s documentation for more info:
<a class="reference external" href="https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html">https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html</a>. If you have a Jetson device
such as a TX1, TX2, Xavier, or Nano, TensorRT will already be installed on the device via the
JetPack SDK.</p>
<p>There are two methods to install TensorRT:</p>
<ul class="simple">
<li><p>System install via deb or rpm package.</p></li>
<li><p>Tar file installation.</p></li>
</ul>
<p>With the tar file installation method, you must provide the path of the extracted tar archive to
USE_TENSORRT_RUNTIME=/path/to/TensorRT. With the system install method,
USE_TENSORRT_RUNTIME=ON will automatically locate your installation.</p>
</div>
<div class="section" id="building-tvm-with-tensorrt-support">
<h2>Building TVM with TensorRT support<a class="headerlink" href="#building-tvm-with-tensorrt-support" title="永久链接至标题">¶</a></h2>
<p>There are two separate build flags for TensorRT integration in TVM. These flags also enable
cross-compilation: USE_TENSORRT_CODEGEN=ON will also you to build a module with TensorRT support on
a host machine, while USE_TENSORRT_RUNTIME=ON will enable the TVM runtime on an edge device to
execute the TensorRT module. You should enable both if you want to compile and also execute models
with the same TVM build.</p>
<ul class="simple">
<li><p>USE_TENSORRT_CODEGEN=ON/OFF - This flag will enable compiling a TensorRT module, which does not require any
TensorRT library.</p></li>
<li><p>USE_TENSORRT_RUNTIME=ON/OFF/path-to-TensorRT - This flag will enable the TensorRT runtime module.
This will build TVM against the installed TensorRT library.</p></li>
</ul>
<p>Example setting in config.cmake file:</p>
<div class="highlight-cmake notranslate"><div class="highlight"><pre><span></span><span class="nb">set</span><span class="p">(</span><span class="s">USE_TENSORRT_CODEGEN</span> <span class="s">ON</span><span class="p">)</span>
<span class="nb">set</span><span class="p">(</span><span class="s">USE_TENSORRT_RUNTIME</span> <span class="s">/home/ubuntu/TensorRT-7.0.0.11</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="build-and-deploy-resnet-18-with-tensorrt">
<h2>Build and Deploy ResNet-18 with TensorRT<a class="headerlink" href="#build-and-deploy-resnet-18-with-tensorrt" title="永久链接至标题">¶</a></h2>
<p>Create a Relay graph from a MXNet ResNet-18 model.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tvm</span>
<span class="kn">from</span> <span class="nn">tvm</span> <span class="kn">import</span> <span class="n">relay</span>
<span class="kn">import</span> <span class="nn">mxnet</span>
<span class="kn">from</span> <span class="nn">mxnet.gluon.model_zoo.vision</span> <span class="kn">import</span> <span class="n">get_model</span>

<span class="n">dtype</span> <span class="o">=</span> <span class="s2">&quot;float32&quot;</span>
<span class="n">input_shape</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">224</span><span class="p">,</span> <span class="mi">224</span><span class="p">)</span>
<span class="n">block</span> <span class="o">=</span> <span class="n">get_model</span><span class="p">(</span><span class="s1">&#39;resnet18_v1&#39;</span><span class="p">,</span> <span class="n">pretrained</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
<span class="n">mod</span><span class="p">,</span> <span class="n">params</span> <span class="o">=</span> <span class="n">relay</span><span class="o">.</span><span class="n">frontend</span><span class="o">.</span><span class="n">from_mxnet</span><span class="p">(</span><span class="n">block</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;data&#39;</span><span class="p">:</span> <span class="n">input_shape</span><span class="p">},</span> <span class="n">dtype</span><span class="o">=</span><span class="n">dtype</span><span class="p">)</span>
</pre></div>
</div>
<p>Annotate and partition the graph for TensorRT. All ops which are supported by the TensorRT
integration will be marked and offloaded to TensorRT. The rest of the ops will go through the
regular TVM CUDA compilation and code generation.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">tvm.relay.op.contrib.tensorrt</span> <span class="kn">import</span> <span class="n">partition_for_tensorrt</span>
<span class="n">mod</span><span class="p">,</span> <span class="n">config</span> <span class="o">=</span> <span class="n">partition_for_tensorrt</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">params</span><span class="p">)</span>
</pre></div>
</div>
<p>Build the Relay graph, using the new module and config returned by partition_for_tensorrt. The
target must always be a cuda target. <code class="docutils literal notranslate"><span class="pre">partition_for_tensorrt</span></code> will automatically fill out the
required values in the config, so there is no need to modify it - just pass it along to the
PassContext so the values can be read during compilation.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">target</span> <span class="o">=</span> <span class="s2">&quot;cuda&quot;</span>
<span class="k">with</span> <span class="n">tvm</span><span class="o">.</span><span class="n">transform</span><span class="o">.</span><span class="n">PassContext</span><span class="p">(</span><span class="n">opt_level</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">config</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;relay.ext.tensorrt.options&#39;</span><span class="p">:</span> <span class="n">config</span><span class="p">}):</span>
    <span class="n">lib</span> <span class="o">=</span> <span class="n">relay</span><span class="o">.</span><span class="n">build</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">target</span><span class="o">=</span><span class="n">target</span><span class="p">,</span> <span class="n">params</span><span class="o">=</span><span class="n">params</span><span class="p">)</span>
</pre></div>
</div>
<p>Export the module.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">lib</span><span class="o">.</span><span class="n">export_library</span><span class="p">(</span><span class="s1">&#39;compiled.so&#39;</span><span class="p">)</span>
</pre></div>
</div>
<p>Load module and run inference on the target machine, which must be built with
<code class="docutils literal notranslate"><span class="pre">USE_TENSORRT_RUNTIME</span></code> enabled. The first run will take longer because the TensorRT engine will
have to be built.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">dev</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">cuda</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="n">loaded_lib</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">load_module</span><span class="p">(</span><span class="s1">&#39;compiled.so&#39;</span><span class="p">)</span>
<span class="n">gen_module</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">contrib</span><span class="o">.</span><span class="n">graph_executor</span><span class="o">.</span><span class="n">GraphModule</span><span class="p">(</span><span class="n">loaded_lib</span><span class="p">[</span><span class="s1">&#39;default&#39;</span><span class="p">](</span><span class="n">dev</span><span class="p">))</span>
<span class="n">input_data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">input_shape</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">dtype</span><span class="p">)</span>
<span class="n">gen_module</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">input_data</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="partitioning-and-compilation-settings">
<h2>Partitioning and Compilation Settings<a class="headerlink" href="#partitioning-and-compilation-settings" title="永久链接至标题">¶</a></h2>
<p>There are some options which can be configured in <code class="docutils literal notranslate"><span class="pre">partition_for_tensorrt</span></code>.</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">version</span></code> - TensorRT version to target as tuple of (major, minor, patch). If TVM is compiled
with USE_TENSORRT_RUNTIME=ON, the linked TensorRT version will be used instead. The version
will affect which ops can be partitioned to TensorRT.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">use_implicit_batch</span></code> - Use TensorRT implicit batch mode (default true). Setting to false will
enable explicit batch mode which will widen supported operators to include those which modify the
batch dimension, but may reduce performance for some models.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">remove_no_mac_subgraphs</span></code> - A heuristic to improve performance. Removes subgraphs which have
been partitioned for TensorRT if they do not have any multiply-accumulate operations. The removed
subgraphs will go through TVM’s standard compilation instead.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">max_workspace_size</span></code> - How many bytes of workspace size to allow each subgraph to use for
TensorRT engine creation. See TensorRT documentation for more info. Can be overriden at runtime.</p></li>
</ul>
</div>
<div class="section" id="runtime-settings">
<h2>Runtime Settings<a class="headerlink" href="#runtime-settings" title="永久链接至标题">¶</a></h2>
<p>There are some additional options which can be configured at runtime using environment variables.</p>
<ul class="simple">
<li><p>Automatic FP16 Conversion - Environment variable <code class="docutils literal notranslate"><span class="pre">TVM_TENSORRT_USE_FP16=1</span></code> can be set to
automatically convert the TensorRT components of your model to 16-bit floating point precision.
This can greatly increase performance, but may cause some slight loss in the model accuracy.</p></li>
<li><p>Caching TensorRT Engines - During the first inference, the runtime will invoke the TensorRT API
to build an engine. This can be time consuming, so you can set <code class="docutils literal notranslate"><span class="pre">TVM_TENSORRT_CACHE_DIR</span></code> to
point to a directory to save these built engines to on the disk. The next time you load the model
and give it the same directory, the runtime will load the already built engines to avoid the long
warmup time. A unique directory is required for each model.</p></li>
<li><p>TensorRT has a paramter to configure the maximum amount of scratch space that each layer in the
model can use. It is generally best to use the highest value which does not cause you to run out
of memory. You can use <code class="docutils literal notranslate"><span class="pre">TVM_TENSORRT_MAX_WORKSPACE_SIZE</span></code> to override this by specifying the
workspace size in bytes you would like to use.</p></li>
<li><p>For models which contain a dynamic batch dimension, the varaible <code class="docutils literal notranslate"><span class="pre">TVM_TENSORRT_MULTI_ENGINE</span></code>
can be used to determine how TensorRT engines will be created at runtime. The default mode,
<code class="docutils literal notranslate"><span class="pre">TVM_TENSORRT_MULTI_ENGINE=0</span></code>, will maintain only one engine in memory at a time. If an input
is encountered with a higher batch size, the engine will be rebuilt with the new max_batch_size
setting. That engine will be compatible with all batch sizes from 1 to max_batch_size. This mode
reduces the amount of memory used at runtime. The second mode, <code class="docutils literal notranslate"><span class="pre">TVM_TENSORRT_MULTI_ENGINE=1</span></code>
will build a unique TensorRT engine which is optimized for each batch size that is encountered.
This will give greater performance, but will consume more memory.</p></li>
</ul>
</div>
<div class="section" id="operator-support">
<h2>Operator support<a class="headerlink" href="#operator-support" title="永久链接至标题">¶</a></h2>
<table class="docutils align-default">
<colgroup>
<col style="width: 40%" />
<col style="width: 60%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>Relay Node</p></th>
<th class="head"><p>Remarks</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>nn.relu</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>sigmoid</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>tanh</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.batch_norm</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>nn.layer_norm</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.softmax</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>nn.conv2d</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.dense</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>nn.bias_add</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>add</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>subtract</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>multiply</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>divide</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>power</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>maximum</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>minimum</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>nn.max_pool2d</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.avg_pool2d</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>nn.global_max_pool2d</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.global_avg_pool2d</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>exp</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>log</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>sqrt</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>abs</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>negative</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.batch_flatten</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>expand_dims</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>squeeze</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>concatenate</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.conv2d_transpose</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>transpose</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>layout_transform</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>reshape</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.pad</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>sum</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>prod</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>max</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>min</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>mean</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.adaptive_max_pool2d</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>nn.adaptive_avg_pool2d</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>nn.batch_matmul</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>clip</p></td>
<td><p>Requires TensorRT 5.1.5 or greater</p></td>
</tr>
<tr class="row-odd"><td><p>nn.leaky_relu</p></td>
<td><p>Requires TensorRT 5.1.5 or greater</p></td>
</tr>
<tr class="row-even"><td><p>sin</p></td>
<td><p>Requires TensorRT 5.1.5 or greater</p></td>
</tr>
<tr class="row-odd"><td><p>cos</p></td>
<td><p>Requires TensorRT 5.1.5 or greater</p></td>
</tr>
<tr class="row-even"><td><p>atan</p></td>
<td><p>Requires TensorRT 5.1.5 or greater</p></td>
</tr>
<tr class="row-odd"><td><p>ceil</p></td>
<td><p>Requires TensorRT 5.1.5 or greater</p></td>
</tr>
<tr class="row-even"><td><p>floor</p></td>
<td><p>Requires TensorRT 5.1.5 or greater</p></td>
</tr>
<tr class="row-odd"><td><p>strided_slice</p></td>
<td><p>Requires TensorRT 5.1.5 or greater</p></td>
</tr>
<tr class="row-even"><td><p>nn.conv3d</p></td>
<td><p>Requires TensorRT 6.0.1 or greater</p></td>
</tr>
<tr class="row-odd"><td><p>nn.max_pool3d</p></td>
<td><p>Requires TensorRT 6.0.1 or greater</p></td>
</tr>
<tr class="row-even"><td><p>nn.avg_pool3d</p></td>
<td><p>Requires TensorRT 6.0.1 or greater</p></td>
</tr>
<tr class="row-odd"><td><p>nn.conv3d_transpose</p></td>
<td><p>Requires TensorRT 6.0.1 or greater</p></td>
</tr>
<tr class="row-even"><td><p>erf</p></td>
<td><p>Requires TensorRT 7.0.0 or greater</p></td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="adding-a-new-operator">
<h2>Adding a new operator<a class="headerlink" href="#adding-a-new-operator" title="永久链接至标题">¶</a></h2>
<p>To add support for a new operator, there are a series of files we need to make changes to:</p>
<ul class="simple">
<li><p><cite>src/runtime/contrib/tensorrt/tensorrt_ops.cc</cite> Create a new op converter class which
implements the <code class="docutils literal notranslate"><span class="pre">TensorRTOpConverter</span></code> interface. You must implement the constructor to specify how
many inputs there are and whether they are tensors or weights. You must also implement the
<code class="docutils literal notranslate"><span class="pre">Convert</span></code> method to perform the conversion. This is done by using the inputs, attributes, and
network from params to add the new TensorRT layers and push the layer outputs. You can use the
existing converters as an example. Finally, register your new op conventer in the
<code class="docutils literal notranslate"><span class="pre">GetOpConverters()</span></code> map.</p></li>
<li><p><cite>python/relay/op/contrib/tensorrt.py</cite> This file contains the annotation rules for TensorRT. These
determine which operators and their attributes that are supported. You must register an annotation
function for the relay operator and specify which attributes are supported by your converter, by
checking the attributes are returning true or false.</p></li>
<li><p><cite>tests/python/contrib/test_tensorrt.py</cite> Add unit tests for the given operator.</p></li>
</ul>
</div>
</div>


           </div>
           
          </div>
          

<footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="vitis_ai.html" class="btn btn-neutral float-right" title="Vitis AI Integration" accesskey="n" rel="next">下一个 <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="arm_compute_lib.html" class="btn btn-neutral float-left" title="Relay Arm® Compute Library Integration" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> 上一个</a>
      
    </div>

<div id="button" class="backtop"><img src="../../_static//img/right.svg" alt="backtop"/> </div>
<section class="footerSec">
    <div class="footerHeader">
      <ul class="d-flex align-md-items-center justify-content-between flex-column flex-md-row">
        <li class="copywrite d-flex align-items-center">
          <h5 id="copy-right-info">© 2020 Apache Software Foundation | All right reserved</h5>
        </li>
      </ul>

    </div>

    <ul>
      <li class="footernote">Copyright © 2020 The Apache Software Foundation. Apache TVM, Apache, the Apache feather, and the Apache TVM project logo are either trademarks or registered trademarks of the Apache Software Foundation.</li>
    </ul>

</section>
</footer>
        </div>
      </div>

    </section>

  </div>
  

    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>

  </body>
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
    <!-- Theme Analytics -->
    <script>
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-75982049-2', 'auto');
    ga('send', 'pageview');
    </script>

    
   

</body>
</html>